
def convert_example_to_feature(example, tokenizer, label_vocab=None, max_seq_len=512, no_entity_label="O", ignore_label=-1, is_test=False):
    tokens, labels = example
    tokenized_input = tokenizer(
        tokens,
        return_length=True,
        is_split_into_words=True,
        max_seq_len=max_seq_len)

    input_ids = tokenized_input['input_ids']
    token_type_ids = tokenized_input['token_type_ids']
    seq_len = tokenized_input['seq_len']

    if is_test:
        return input_ids, token_type_ids, seq_len
    elif label_vocab is not None:
        labels = labels[:(max_seq_len-2)]
        encoded_label = [no_entity_label] + labels + [no_entity_label]
        encoded_label = [label_vocab[x] for x in encoded_label]
        return input_ids, token_type_ids, seq_len, encoded_label


def parse_result(words, preds, lengths, sentences, label_vocab):
    """ parse padding result """
    batch_out = []
    # id2word_dict = dict(zip(word_vocab.values(), word_vocab.keys()))
    id2label_dict = dict(zip(label_vocab.values(), label_vocab.keys()))
    for sent_index in range(len(lengths)):
        sent = list(sentences[sent_index])

        tags = [
            id2label_dict[index]
            for index in preds[sent_index][:lengths[sent_index]]
        ]

        sent_out = []
        tags_out = []
        pos_out = []
        pos = []
        parital_word = ""
        for ind, tag in enumerate(tags):
            if tag.startswith("B-"):
                tags_out.append(tag.split('-')[1])
                if len(pos) == 0:
                    pos.append(ind)
                else:
                    pos_out.append(pos)
                    pos = []
                    pos.append(ind)
            if tag == "O" and tags[ind - 1] != "O":
                pos.append(ind - 1)
            # # for the first word
            # if parital_word == "":
            #     parital_word = sent[ind]
            #     if tag != 'O':
            #         tags_out.append(tag.split('-')[1])
            #     continue

            # for the beginning of word
            # if tag.startswith("B-") or (tag == "O" and tags[ind - 1] != "O"):
            #     sent_out.append(parital_word)
            #     tags_out.append(tag.split('-')[1])
            #     parital_word = sent[ind]
            #     continue

            # parital_word += sent[ind]

        # append the last word, except for len(tags)=0
        # if len(sent_out) < len(tags_out):
        #     sent_out.append(parital_word)

        batch_out.append([sent_out, tags_out, pos_out])
    return batch_out